package com.ningxun.pagentcrawler.util;

import com.ningxun.pagentcrawler.entity.policy.PolicyItem;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.util.List;
import java.util.Set;

/**
 * 爬虫工具类
 * 负责爬虫的主要流程控制
 */
public class CrawlerUtil {

    /**
     * 使用Jsoup爬取多页网站数据（新版，适配Ajax分页）
     */
    public static void crawlMultiplePagesWithJsoup(List<PolicyItem> crawledPolicies, Set<String> urlSet) {
        try {
            System.out.println("正在使用Jsoup爬取河北省政府网站多页数据（Ajax分页）...");
            String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36";

            // 配置SSL证书验证忽略
            configureSSL();

            // 先请求第一页，获取总页数
            int totalPages = PageParser.getTotalPages(userAgent);
            System.out.println("共 " + totalPages + " 页");

            for (int currentPage = 1; currentPage <= totalPages; currentPage++) {
                try {
                    System.out.println("正在爬取第 " + currentPage + " 页...");
                    String pageUrl = UrlBuilder.buildAjaxPageUrl(currentPage);
                    Document doc = Jsoup.connect(pageUrl)
                            .userAgent(userAgent)
                            .timeout(15000)
                            .get();
                    int pageCount = PageParser.parsePolicyListFromAjaxPage(doc, currentPage, crawledPolicies, urlSet);
                    System.out.println("✅ 第 " + currentPage + " 页爬取完成，找到 " + pageCount + " 条政策数据");
                    Thread.sleep(2000);
                } catch (Exception e) {
                    System.err.println("❌ 爬取第 " + currentPage + " 页失败: " + e.getMessage());
                    continue;
                }
            }
            System.out.println("✅ 多页爬虫完成！总共找到 " + crawledPolicies.size() + " 条政策数据");
        } catch (Exception e) {
            System.err.println("❌ Jsoup多页爬虫失败: " + e.getMessage());
            e.printStackTrace();
        }
    }

    /**
     * 配置SSL证书验证忽略
     */
    public static void configureSSL() {
        System.setProperty("https.protocols", "TLSv1.2");
        System.setProperty("javax.net.ssl.trustStore", "");
        System.setProperty("javax.net.ssl.trustStorePassword", "");
        
        // 设置忽略SSL证书验证
        javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
        
        try {
            // 创建信任所有证书的TrustManager
            javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[]{
                new javax.net.ssl.X509TrustManager() {
                    public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; }
                    public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { }
                    public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { }
                }
            };

            // 创建SSLContext并设置TrustManager
            javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance("SSL");
            sc.init(null, trustAllCerts, new java.security.SecureRandom());
            javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
            
        } catch (Exception e) {
            System.err.println("SSL配置失败: " + e.getMessage());
        }
    }
} 